#!/usr/bin/python
# -*- coding:utf-8 -*-
import StringIO
import gzip
import socket
import ssl
import urllib2
import ConfigParser
from ext import logging
import requests

class Downloader(object):
    def __init__(self):
        cf = ConfigParser.ConfigParser()
        cf.read('config.ini')
        self.headers = [{
            'Accpet': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Cookie': cf.get('opener_1', 'cookie'),
            'Host': cf.get('opener_1', 'host'),
            'User-Agent': cf.get('opener_1', 'user-agent'),
            'Accept-Encoding': 'gzip'
        }, {
            'Accpet': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Cookie': cf.get('opener_2', 'cookie'),
            'Host': cf.get('opener_2', 'host'),
            'User-Agent': cf.get('opener_2', 'user-agent'),
            'Accept-Encoding': 'gzip'
        }]
        opener = urllib2.build_opener(urllib2.ProxyHandler())
        urllib2.install_opener(opener)

    def do_download(self, url):
        """
        根据网页地址下载网页
        :param self:
        :param url: 网页地址
        :return: 爬取成功以字符串方式返回网页代码，爬取失败返回None
        """
        if url is None:
            return None
        try:
            request = urllib2.Request(url=url, headers=self.headers[0])
            if '.htm' in url:
                request = urllib2.Request(url=url, headers=self.headers[1])
            response = urllib2.urlopen(request, timeout=15)

            if response.getcode() == 200 and response.info().get('Content-Encoding') == 'gzip':
                content = response.read()
                data = StringIO.StringIO(content)
                gz = gzip.GzipFile(fileobj=data)
                content = gz.read()
                gz.close()
                return content
            return None
        except urllib2.HTTPError, e1:
            print u'open failed，address:%s,code:%d' % (url, e1.code)
            return None
        except urllib2.URLError, e2:
            print u'open failed，info：%s' % e2
            return None
        except ssl.SSLError, socket.error:
            logging.info(u'timed out.')
            return self.do_download(url)
        except Exception:
            logging.error(u'未知异常')
            return None
